{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\IRIDES~1\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.971 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "        <script type=\"text/javascript\">\n",
       "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
       "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
       "        if (typeof require !== 'undefined') {\n",
       "        require.undef(\"plotly\");\n",
       "        requirejs.config({\n",
       "            paths: {\n",
       "                'plotly': ['https://cdn.plot.ly/plotly-2.9.0.min']\n",
       "            }\n",
       "        });\n",
       "        require(['plotly'], function(Plotly) {\n",
       "            window._Plotly = Plotly;\n",
       "        });\n",
       "        }\n",
       "        </script>\n",
       "        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import chart_studio\n",
    "import plotly.graph_objs as grho\n",
    "\n",
    "# Cufflinks wrapper on plotly\n",
    "import cufflinks\n",
    "\n",
    "# Data science imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "from excavator import TextFormer\n",
    "from cleaner import CleanedCorpuses\n",
    "\n",
    "chart_studio.tools.set_credentials_file(username='she_xi', api_key='n1rSTfNaLYD5w3X9yy60')\n",
    "\n",
    "from plotly.offline import iplot\n",
    "cufflinks.go_offline()\n",
    "\n",
    "from mlxtend.preprocessing import TransactionEncoder\n",
    "from mlxtend.frequent_patterns import apriori, association_rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./input_data/artists.txt', 'r', encoding='utf-8') as f:\n",
    "    artists = f.readlines()\n",
    "    artists = [i.strip(' \\n').split(' ')[0] for i in artists]\n",
    "with open('./input_data/repertory.txt', 'r', encoding='utf-8') as f:\n",
    "    repertory = f.readlines()\n",
    "    repertory = [i.strip(' \\n').split(' ')[0] for i in repertory]\n",
    "with open('./input_data/cliques.txt', 'r', encoding='utf-8') as f:\n",
    "    cliques = f.readlines()\n",
    "    cliques = [i.strip(' \\n').split(' ')[0] for i in cliques]\n",
    "with open('./input_data/vocations.txt', 'r', encoding='utf-8') as f:\n",
    "    vocations = f.readlines()\n",
    "    vocations = [i.strip(' \\n').split(' ')[0] for i in vocations]\n",
    "\n",
    "key_words = { 'artists':artists, 'repertory':repertory, 'cliques':cliques, 'vocations':vocations }\n",
    "\n",
    "terms = artists + repertory + cliques + vocations\n",
    "\n",
    "target_tags = [\n",
    "    'n', 'nr', 'ns', 'nt',\n",
    "    'nw', 'nz', 'PER', 'LOC', 'ORG'\n",
    "]\n",
    "\n",
    "paths = {\n",
    "    'artist_info':'./input_data/BaiduBaike/artists_info.csv', \n",
    "    'repertory_info':'./input_data/BaiduBaike/repertory_info.csv',\n",
    "    'bilibili_comment':'./input_data/Bilibili/jingju_comment.csv',\n",
    "    'bilibili_cleaned_by_oid':'./input_data/Bilibili/bili_cleaned_msg_by_oid.csv',\n",
    "    'bilibili_cleaned_by_uname':'./input_data/Bilibili/bili_cleaned_msg_by_uname.csv'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def AssoRuleData(corpus:list, key_words:list, word_seperator:str = ' ') -> list:\n",
    "    key_words_set = set(key_words)\n",
    "    data = [i.split(' ') for i in corpus]\n",
    "    data = [list(set(i)&key_words_set) for i in data]\n",
    "    return [i for i in data if i != []]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_origin = pd.read_csv(paths['bilibili_cleaned_by_uname'])\n",
    "df_origin = df_origin.fillna(' ')\n",
    "cleaned = list(df_origin['message'])\n",
    "data = AssoRuleData(cleaned, terms)\n",
    "\n",
    "df_origin_2 = pd.read_csv(paths['bilibili_cleaned_by_oid'])\n",
    "df_origin_2 = df_origin_2.fillna(' ')\n",
    "cleaned_2 = list(df_origin_2['message'])\n",
    "data_2 = AssoRuleData(cleaned_2, terms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "Encoder = TransactionEncoder()\n",
    "encoded_data = Encoder.fit_transform(data)\n",
    "df = pd.DataFrame(encoded_data, columns=Encoder.columns_)\n",
    "\n",
    "Encoder_2 = TransactionEncoder()\n",
    "encoded_data_2 = Encoder_2.fit_transform(data_2)\n",
    "df_2 = pd.DataFrame(encoded_data_2, columns=Encoder_2.columns_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_165956\\2023157181.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mfrequent_items\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mapriori\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_support\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.005\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_colnames\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'support'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mfrequent_items_2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mapriori\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf_2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmin_support\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.005\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_colnames\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_len\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'support'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\mlxtend\\frequent_patterns\\apriori.py\u001b[0m in \u001b[0;36mapriori\u001b[1;34m(df, min_support, use_colnames, max_len, verbose, low_memory)\u001b[0m\n\u001b[0;32m    285\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    286\u001b[0m             \u001b[0mcombin\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgenerate_new_combinations\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitemset_dict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmax_itemset\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 287\u001b[1;33m             \u001b[0mcombin\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromiter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcombin\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    288\u001b[0m             \u001b[0mcombin\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcombin\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnext_max_itemset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    289\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\mlxtend\\frequent_patterns\\apriori.py\u001b[0m in \u001b[0;36mgenerate_new_combinations\u001b[1;34m(old_combinations)\u001b[0m\n\u001b[0;32m     49\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mitem\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mvalid_items\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m             \u001b[1;32myield\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mold_tuple\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m             \u001b[1;32myield\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "frequent_items = apriori(df, min_support=0.005, use_colnames=True).sort_values(by='support', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>antecedents</th>\n",
       "      <th>consequents</th>\n",
       "      <th>antecedent support</th>\n",
       "      <th>consequent support</th>\n",
       "      <th>support</th>\n",
       "      <th>confidence</th>\n",
       "      <th>lift</th>\n",
       "      <th>leverage</th>\n",
       "      <th>conviction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(老旦)</td>\n",
       "      <td>(青衣)</td>\n",
       "      <td>0.047468</td>\n",
       "      <td>0.025316</td>\n",
       "      <td>0.006329</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>5.266667</td>\n",
       "      <td>0.005127</td>\n",
       "      <td>1.124635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(青衣)</td>\n",
       "      <td>(老旦)</td>\n",
       "      <td>0.025316</td>\n",
       "      <td>0.047468</td>\n",
       "      <td>0.006329</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>5.266667</td>\n",
       "      <td>0.005127</td>\n",
       "      <td>1.270042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(梨花颂)</td>\n",
       "      <td>(梅葆玖)</td>\n",
       "      <td>0.029747</td>\n",
       "      <td>0.040506</td>\n",
       "      <td>0.006329</td>\n",
       "      <td>0.212766</td>\n",
       "      <td>5.252660</td>\n",
       "      <td>0.005124</td>\n",
       "      <td>1.218816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(梅葆玖)</td>\n",
       "      <td>(梨花颂)</td>\n",
       "      <td>0.040506</td>\n",
       "      <td>0.029747</td>\n",
       "      <td>0.006329</td>\n",
       "      <td>0.156250</td>\n",
       "      <td>5.252660</td>\n",
       "      <td>0.005124</td>\n",
       "      <td>1.149930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(程派)</td>\n",
       "      <td>(青衣)</td>\n",
       "      <td>0.050633</td>\n",
       "      <td>0.025316</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>0.112500</td>\n",
       "      <td>4.443750</td>\n",
       "      <td>0.004414</td>\n",
       "      <td>1.098235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(青衣)</td>\n",
       "      <td>(程派)</td>\n",
       "      <td>0.025316</td>\n",
       "      <td>0.050633</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>0.225000</td>\n",
       "      <td>4.443750</td>\n",
       "      <td>0.004414</td>\n",
       "      <td>1.224990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(铡美案)</td>\n",
       "      <td>(秦香莲)</td>\n",
       "      <td>0.029747</td>\n",
       "      <td>0.065823</td>\n",
       "      <td>0.005063</td>\n",
       "      <td>0.170213</td>\n",
       "      <td>2.585925</td>\n",
       "      <td>0.003105</td>\n",
       "      <td>1.125803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(锁麟囊)</td>\n",
       "      <td>(程派)</td>\n",
       "      <td>0.051899</td>\n",
       "      <td>0.050633</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>0.109756</td>\n",
       "      <td>2.167683</td>\n",
       "      <td>0.003068</td>\n",
       "      <td>1.066412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(程派)</td>\n",
       "      <td>(锁麟囊)</td>\n",
       "      <td>0.050633</td>\n",
       "      <td>0.051899</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>0.112500</td>\n",
       "      <td>2.167683</td>\n",
       "      <td>0.003068</td>\n",
       "      <td>1.068283</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  antecedents consequents  antecedent support  consequent support   support  \\\n",
       "2        (老旦)        (青衣)            0.047468            0.025316  0.006329   \n",
       "3        (青衣)        (老旦)            0.025316            0.047468  0.006329   \n",
       "0       (梨花颂)       (梅葆玖)            0.029747            0.040506  0.006329   \n",
       "1       (梅葆玖)       (梨花颂)            0.040506            0.029747  0.006329   \n",
       "6        (程派)        (青衣)            0.050633            0.025316  0.005696   \n",
       "7        (青衣)        (程派)            0.025316            0.050633  0.005696   \n",
       "8       (铡美案)       (秦香莲)            0.029747            0.065823  0.005063   \n",
       "4       (锁麟囊)        (程派)            0.051899            0.050633  0.005696   \n",
       "5        (程派)       (锁麟囊)            0.050633            0.051899  0.005696   \n",
       "\n",
       "   confidence      lift  leverage  conviction  \n",
       "2    0.133333  5.266667  0.005127    1.124635  \n",
       "3    0.250000  5.266667  0.005127    1.270042  \n",
       "0    0.212766  5.252660  0.005124    1.218816  \n",
       "1    0.156250  5.252660  0.005124    1.149930  \n",
       "6    0.112500  4.443750  0.004414    1.098235  \n",
       "7    0.225000  4.443750  0.004414    1.224990  \n",
       "8    0.170213  2.585925  0.003105    1.125803  \n",
       "4    0.109756  2.167683  0.003068    1.066412  \n",
       "5    0.112500  2.167683  0.003068    1.068283  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ass_rule = association_rules(frequent_items, metric='confidence', min_threshold=0.1)\n",
    "ass_rule.sort_values(by='leverage', ascending=False, inplace=True)\n",
    "ass_rule"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
